import scrapy
from scrapy_redis.spiders import RedisSpider

class CommentSpider(RedisSpider):
    name = "comment"
    allowed_domains = ["douban.com"]
    start_urls = ["https://movie.douban.com/review/best/"]
    redis_key = 'DouBan'

    def __init__(self, *args, **kwargs):
        domain = kwargs.pop('domain', '')
        self.allowed_domains = list(filter(None, domain.split(',')))
        super(CommentSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        link_node_list = response.xpath('//div/h2/a')

        for link_node in link_node_list:
            temp = {}
            comment_name = link_node.xpath('./text()').extract_first()
            comment_link = link_node.xpath('./@href').extract_first()
            temp['comment_name'] = comment_name
            temp['comment_link'] = comment_link

            yield scrapy.Request(
                url=comment_link,
                callback=self.comment_solve,
                meta={'meta1': temp}
            )

        part_url = response.xpath('//div/div[1]/div[2]/span[3]/a/@href').extract_first()
        if part_url and part_url != '/review/best?start=60':
            next_url = response.urljoin(part_url)
            yield scrapy.Request(
                url=next_url,
                callback=self.parse
            )

    def comment_solve(self, response):
        item = response.meta['meta1']

        movie_link = response.xpath('//header/a[2]/@href').extract_first()

        comment_xpath = '//div[1]/p[position() < 4]'
        comments = response.xpath(comment_xpath)
        comment_content = ' '.join(comment.xpath('string(.)').get().strip() for comment in comments)

        item['movie_link'] = movie_link
        item['comment_content'] = comment_content

        yield scrapy.Request(
            url=movie_link,
            callback=self.movie_solve,
            meta={'meta2': item}
        )

    def movie_solve(self, response):
        print(response)
        item = response.meta['meta2']

        movie_name = response.xpath('//h1/span[1]/text()').extract_first()
        movie_rate = response.xpath('//div[1]/div[2]/strong/text()').extract_first()
        movie_director = response.xpath('//span[1]/span[2]/a/text()').extract_first()

        movie_type = ' / '.join(response.css('span[property="v:genre"]::text').getall())
        movie_time = response.css('span[property="v:runtime"]::text').extract_first()

        info = response.css('#info')
        movie_country =  self.extract_with_colon(info, '制片国家/地区')
        movie_language = self.extract_with_colon(info, '语言')

        item['movie_name'] = movie_name
        item['movie_rate'] = movie_rate
        item['movie_director'] = movie_director
        item['movie_type'] = movie_type
        item['movie_time'] = movie_time
        item['movie_country'] = movie_country
        item['movie_language'] = movie_language
        item['kind'] = 'DouBan'

        yield item

    @staticmethod
    def extract_with_colon(info_section, item_name):
        # 辅助函数，用于提取带有冒号的信息，如“制片国家/地区:”
        item = info_section.xpath(f'.//span[contains(text(), "{item_name}")]/following-sibling::text()').get()
        if item:
            return item.strip()
        return None